{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dataclasses import dataclass, field\n",
    "import gc\n",
    "import glob\n",
    "import logging\n",
    "import math\n",
    "import os\n",
    "import pandas as pd\n",
    "import pyarrow.feather as feather\n",
    "from typing import Optional\n",
    "from tqdm.contrib.concurrent import thread_map\n",
    "from tqdm import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data.dataset import Dataset\n",
    "\n",
    "from transformers import (\n",
    "    MODEL_WITH_LM_HEAD_MAPPING,\n",
    "    CONFIG_MAPPING,\n",
    "    AutoConfig,\n",
    "    AutoModelWithLMHead,\n",
    "    AutoTokenizer,\n",
    "    DataCollatorForLanguageModeling,\n",
    "    DataCollatorForPermutationLanguageModeling,\n",
    "    HfArgumentParser,\n",
    "    LineByLineTextDataset,\n",
    "    PreTrainedTokenizer,\n",
    "    TextDataset,\n",
    "    Trainer,\n",
    "    TrainingArguments,\n",
    "    set_seed,\n",
    "    BertTokenizerFast\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizerFast('/home/ajay/contracode/data/vocab/8k_bpe/8k_bpe-vocab.txt', clean_text=True, lowercase=False, strip_accents=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer.pad_token_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
